{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "While converting to CoreML there is an option to set image preprocessing parameters. Channel wise bias and an overall scale is supported, which is quite common. However, some models may require a per channel scale parameter. \n",
    "This can be implemented by adding a \"scale\" layer in the beginning of the network, after conversion. Let us see how this can be done by directly editing the mlmodel spec which is in protobuf format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import coremltools\n",
    "from keras.layers import *\n",
    "from keras.models import Sequential\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 : cropping2d_3_input, <keras.engine.topology.InputLayer object at 0x12b310090>\n",
      "1 : cropping2d_3, <keras.layers.convolutional.Cropping2D object at 0x12825b390>\n"
     ]
    }
   ],
   "source": [
    "# Define a toy Keras network and convert to CoreML\n",
    "input_shape = (50, 50, 3)\n",
    "model = Sequential()\n",
    "model.add(Cropping2D(cropping=((5,5),(5,5)), input_shape=input_shape))\n",
    "\n",
    "\n",
    "mlmodel = coremltools.converters.keras.convert(model,\n",
    "                                              image_input_names='input1',\n",
    "                                              red_bias=-10.0, \n",
    "                                              green_bias=-10.0, \n",
    "                                              blue_bias=-10.0,\n",
    "                                              image_scale=5.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input {\n",
      "  name: \"input1\"\n",
      "  type {\n",
      "    imageType {\n",
      "      width: 50\n",
      "      height: 50\n",
      "      colorSpace: RGB\n",
      "    }\n",
      "  }\n",
      "}\n",
      "output {\n",
      "  name: \"output1\"\n",
      "  type {\n",
      "    multiArrayType {\n",
      "      shape: 3\n",
      "      shape: 40\n",
      "      shape: 40\n",
      "      dataType: DOUBLE\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spec = mlmodel.get_spec()\n",
    "print(spec.description)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('output along channel at [0,0]: ', array([490., 490., 490.]))\n"
     ]
    }
   ],
   "source": [
    "# Lets call predict with an all constant image input\n",
    "x = 100.0 * np.ones((3,50,50))\n",
    "x = x.astype(np.uint8)\n",
    "x_transpose = np.transpose(x, [1,2,0]) # PIL Image requires the format to be [H,W,C]\n",
    "im = Image.fromarray(x_transpose)\n",
    "\n",
    "y = mlmodel.predict({'input1': im}, useCPUOnly=True)['output1']\n",
    "print('output along channel at [0,0]: ', y[:,0,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected the output values are 490. That is, ${\\textrm {scale}} * {\\textrm {input}} + {\\textrm {bias}}, \\;\\;{\\textrm{i.e.,}}\\;\\;5*100 -10 = 490$.\n",
    "Let us insert a channel dependent scale layer in the beginning of the network, before the crop layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get NN portion of the spec\n",
    "nn_spec = spec.neuralNetwork\n",
    "layers = nn_spec.layers # this is a list of all the layers\n",
    "layers_copy = copy.deepcopy(layers) # make a copy of the layers, these will be added back later\n",
    "del nn_spec.layers[:] # delete all the layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add a scale layer now\n",
    "# since mlmodel is in protobuf format, we can add proto messages directly\n",
    "# To look at more examples on how to add other layers: see \"builder.py\" file in coremltools repo\n",
    "scale_layer = nn_spec.layers.add()\n",
    "scale_layer.name = 'scale_layer'\n",
    "scale_layer.input.append('input1')\n",
    "scale_layer.output.append('input1_scaled')\n",
    "params = scale_layer.scale\n",
    "params.scale.floatValue.extend([1.0, 2.0, 3.0]) # scale values for RGB\n",
    "params.shapeScale.extend([3,1,1]) # shape of the scale vector \n",
    "\n",
    "# now add back the rest of the layers (which happens to be just one in this case: the crop layer)\n",
    "nn_spec.layers.extend(layers_copy)\n",
    "\n",
    "# need to also change the input of the crop layer to match the output of the scale layer\n",
    "nn_spec.layers[1].input[0] = 'input1_scaled'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('output along channel at [0,0]: ', array([ 490.,  980., 1470.]))\n"
     ]
    }
   ],
   "source": [
    "# Lets run the model again\n",
    "mlmodel = coremltools.models.MLModel(spec)\n",
    "y = mlmodel.predict({'input1': im}, useCPUOnly=True)['output1']\n",
    "print('output along channel at [0,0]: ', y[:,0,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected the values are scaled by 1.0, 2.0, 3.0: the parameters of the scale layer. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
